library(ggplot2)
library(tidyverse)
── Attaching core tidyverse packages ───────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr 1.1.4 ✔ readr 2.1.5
✔ forcats 1.0.0 ✔ stringr 1.5.1
✔ lubridate 1.9.4 ✔ tibble 3.2.1
✔ purrr 1.0.2 ✔ tidyr 1.3.1── Conflicts ─────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag() masks stats::lag()
ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors
library(car)
Loading required package: carData
Attaching package: ‘car’
The following object is masked from ‘package:dplyr’:
recode
The following object is masked from ‘package:purrr’:
some
library(mgcv)
Loading required package: nlme
Attaching package: ‘nlme’
The following object is masked from ‘package:dplyr’:
collapse
This is mgcv 1.9-1. For overview type 'help("mgcv-package")'.
require(gam)
Loading required package: gam
Loading required package: splines
Loading required package: foreach
Attaching package: ‘foreach’
The following objects are masked from ‘package:purrr’:
accumulate, when
Loaded gam 1.22-5
Attaching package: ‘gam’
The following objects are masked from ‘package:mgcv’:
gam, gam.control, gam.fit, s
require(lmtest)
Loading required package: lmtest
Loading required package: zoo
Attaching package: ‘zoo’
The following objects are masked from ‘package:base’:
as.Date, as.Date.numeric
require(lme4)
Loading required package: lme4
Loading required package: Matrix
Attaching package: ‘Matrix’
The following objects are masked from ‘package:tidyr’:
expand, pack, unpack
Attaching package: ‘lme4’
The following object is masked from ‘package:nlme’:
lmList
df<-read.csv("replication_final.csv")
df$Family = str_to_title(str_replace(df$Family, "_", " "))
CHRTRM <- df %>%
select(c(Language,
Family,
SIGMORPHON_acc = CHR.TRM_SIGMORPHON_test_acc,
Goldman_acc = CHR.TRM_Goldman_test_acc,
Goldman_train_size,
Goldman_train_lemmas,
train_lemma_diff_raw,
SIGMORPHON_train_size,
SIGMORPHON_train_lemmas
))
CHRTRM$Model = "CHR-TRM"
CLUZH <- df %>%
select(c(Language,
Family,
SIGMORPHON_acc = CLUZH_SIGMORPHON_test_acc,
Goldman_acc = CLUZH_Goldman_test_acc,
Goldman_train_size,
Goldman_train_lemmas,
train_lemma_diff_raw,
SIGMORPHON_train_size,
SIGMORPHON_train_lemmas
))
CLUZH$Model = "CLUZH"
DeepSpin <- df %>%
select(c(Language,
Family,
SIGMORPHON_acc = DeepSpin_SIGMORPHON_test_acc,
Goldman_acc = DeepSpin_Goldman_test_acc,
Goldman_train_size,
Goldman_train_lemmas,
train_lemma_diff_raw,
SIGMORPHON_train_size,
SIGMORPHON_train_lemmas
))
DeepSpin$Model = "DeepSpin"
all_data <- rbind(CLUZH, CHRTRM, DeepSpin)
all_data$acc_drop = all_data$Goldman_acc - all_data$SIGMORPHON_acc
all_data$log_acc_drop <- -1*log(-1*(all_data$acc_drop - 4))
max(all_data$acc_drop)
[1] 2.0202
# We'll use this helper function to run our stats
correlations <- function(a, b){
for (m in list("pearson", "spearman", "kendall")){
# Supressing warnings bc we'll get them whenever there are ties
suppressWarnings(res <- cor.test(a, b, method = m))
formatted <- sprintf("%s: %f (p = %f)", res$method, res$estimate, res$p.value)
print(formatted)
}
}
# We'll use this helper function to get information about our model
eval_model <- function(model, df){
rsquared = summary(model)$r.squared
AIC = AIC(model)
results <- sprintf("R^2: %f, AIC: %f", rsquared, AIC)
print(results)
layout(matrix(c(1,2,3,4),2,2))
plot(model)
return(predict(model, df, se = TRUE))
}
# We'll use this helper function to compare two models
compare_models <- function(model_both, model_single){
layout(matrix(c(1,2,3,4),2,2))
plot(model_both)
AIC = AIC(model_both)
anovap = anova(model_both, model_single)$`Pr(>F)`[-1]
results <- sprintf("AIC: %f, ANOVA p: %f", AIC, anovap)
print(results)
vif(model_both)
}
SIGMORPHON <- all_data %>%
select(c(Model, Family, train_size=SIGMORPHON_train_size, train_lemmas=SIGMORPHON_train_lemmas, test_acc = SIGMORPHON_acc))
SIGMORPHON$Data <- "SIGMORPHON"
Goldman <- all_data %>%
select(c(Model, Family, train_size=Goldman_train_size, train_lemmas=Goldman_train_lemmas, test_acc = Goldman_acc))
Goldman$Data <- "Goldman et al."
rbind(SIGMORPHON, Goldman) %>%
ggplot(aes(log(train_size), log(test_acc), color=Data)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
scale_color_manual(values=c("turquoise", "purple")) +
geom_point(aes(shape=Family), size=3, alpha=0.6) +
theme_bw() +
xlab("Training Size, Log Scale") +
ylab("Test Accuracy, Log Scale") +
ggtitle("Test Accuracy vs. Training Size") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_size_accuracy.pdf", device="pdf", width = 12, height=4)
rbind(SIGMORPHON, Goldman) %>%
ggplot(aes(log(train_lemmas), log(test_acc), color=Data)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
scale_color_manual(values=c("turquoise", "purple")) +
geom_point(aes(shape=Family), size=3, alpha=0.6) +
theme_bw() +
xlab("Training Lemmas, Log Scale") +
ylab("Test Accuracy, Log Scale") +
ggtitle("Test Accuracy vs. Training Lemmas") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_lemmas_accuracy.pdf", device="pdf", width = 12, height=4)
correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.703670 (p = 0.000003)"
[1] "Spearman's rank correlation rho: 0.817515 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.621986 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.411709 (p = 0.015553)"
[1] "Spearman's rank correlation rho: 0.028615 (p = 0.872373)"
[1] "Kendall's rank correlation tau: -0.012590 (p = 0.917200)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_size), log(CLUZH$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.509641 (p = 0.002079)"
[1] "Spearman's rank correlation rho: 0.062892 (p = 0.723823)"
[1] "Kendall's rank correlation tau: 0.019785 (p = 0.870228)"
correlations(log(CHRTRM$SIGMORPHON_train_size), log(CHRTRM$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.308926 (p = 0.075448)"
[1] "Spearman's rank correlation rho: 0.039633 (p = 0.823893)"
[1] "Kendall's rank correlation tau: 0.008993 (p = 0.940805)"
sprintf("")
[1] ""
correlations(log(DeepSpin$SIGMORPHON_train_size), log(DeepSpin$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.318779 (p = 0.066132)"
[1] "Spearman's rank correlation rho: -0.028388 (p = 0.873380)"
[1] "Kendall's rank correlation tau: -0.058140 (p = 0.633569)"
sprintf("")
[1] ""
correlations(log(CLUZH$SIGMORPHON_train_size), log(CLUZH$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.401506 (p = 0.018593)"
[1] "Spearman's rank correlation rho: 0.120893 (p = 0.495836)"
[1] "Kendall's rank correlation tau: 0.078642 (p = 0.514054)"
correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.663836 (p = 0.000019)"
[1] "Spearman's rank correlation rho: 0.849805 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.646953 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.239558 (p = 0.172389)"
[1] "Spearman's rank correlation rho: -0.011326 (p = 0.949309)"
[1] "Kendall's rank correlation tau: -0.057711 (p = 0.634496)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.381134 (p = 0.026152)"
[1] "Spearman's rank correlation rho: 0.061529 (p = 0.729582)"
[1] "Kendall's rank correlation tau: -0.010821 (p = 0.928971)"
correlations(log(CHRTRM$SIGMORPHON_train_lemmas), log(CHRTRM$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.155920 (p = 0.378556)"
[1] "Spearman's rank correlation rho: -0.002142 (p = 0.990405)"
[1] "Kendall's rank correlation tau: -0.057608 (p = 0.634570)"
sprintf("")
[1] ""
correlations(log(DeepSpin$SIGMORPHON_train_lemmas), log(DeepSpin$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.161117 (p = 0.362669)"
[1] "Spearman's rank correlation rho: -0.084248 (p = 0.635701)"
[1] "Kendall's rank correlation tau: -0.118201 (p = 0.332834)"
sprintf("")
[1] ""
correlations(log(CLUZH$SIGMORPHON_train_lemmas), log(CLUZH$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.268885 (p = 0.124126)"
[1] "Spearman's rank correlation rho: 0.095224 (p = 0.592169)"
[1] "Kendall's rank correlation tau: 0.030411 (p = 0.800929)"
data <- rbind(SIGMORPHON, Goldman)
data$scaled_acc <- (data$test_acc/100 * 203 + .5) / 204
model <- glmer(scaled_acc ~ log(train_lemmas) + log(train_size) + (1|Model) + ( train_size + train_lemmas |Family) + (train_lemmas |Data), data=data, family=binomial)
summary(model)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(x = Goldman_scaled_acc, color=Model)) +
geom_density(aes(fill= Model), alpha = 0.5) +
scale_color_manual(values=c("purple", "gold", "turquoise")) +
scale_fill_manual(values=c("purple", "gold", "turquoise")) +
theme_bw() +
ylab("Density") +
xlab("Goldman et al. Test Accuracy") +
ggtitle("Density of Goldman et al. Test Accuracy") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/accuracy_distribution.pdf", device="pdf", width = 8, height=5)
CHRTRM$Goldman_scaled_acc <- (CHRTRM$Goldman_acc/100 * 33 + .5) / 34
size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(size_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.4786 -0.8088 0.0000 0.2771 1.6071
(Dispersion Parameter for Beta regression family taken to be 0.3493)
Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 1.0581 on 32 degrees of freedom
AIC: -3694.535
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 19.689 19.6892 56.36 1.51e-08 ***
Residuals 32 11.179 0.3493
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit
lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = betar(link = "logit"), data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.9217 -0.8786 0.0000 0.0000 1.1945
(Dispersion Parameter for Beta regression family taken to be 0.3355)
Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 12.528 on 32 degrees of freedom
AIC: -3798.311
Number of Local Scoring Iterations: 6
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 13.168 13.1679 39.247 5.054e-07 ***
Residuals 32 10.736 0.3355
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(both_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.7181 -0.9309 0.0000 0.0000 1.3871
(Dispersion Parameter for Beta regression family taken to be 0.3214)
Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 9.2018 on 31 degrees of freedom
AIC: -3878.832
Number of Local Scoring Iterations: 6
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 14.2329 14.2329 44.2893 1.932e-07 ***
log(Goldman_train_size) 1 0.7122 0.7122 2.2163 0.1467
Residuals 31 9.9622 0.3214
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -3694.535
AIC(lemma_model)
[1] -3798.311
AIC(both_model)
[1] -3878.832
BIC(size_model)
[1] -3691.483
BIC(lemma_model)
[1] -3795.258
BIC(both_model)
[1] -3874.253
size_model$coefficients
(Intercept) log(Goldman_train_size)
-4.9490885 0.6433317
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
-3.8351949 0.8111991
DeepSpin$Goldman_scaled_acc <- (DeepSpin$Goldman_acc/100 * 33 + .5) / 34
size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(size_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.5926 -1.0797 0.0000 0.0000 0.8346
(Dispersion Parameter for Beta regression family taken to be 0.3122)
Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.9009 on 32 degrees of freedom
AIC: -12273.43
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 3.5977 3.5977 11.522 0.00185 **
Residuals 32 9.9918 0.3122
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit
lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(lemma_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = betar(link = "logit"), data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.6147 -1.0699 0.0000 0.0000 0.5571
(Dispersion Parameter for Beta regression family taken to be 0.3817)
Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -17.41 on 32 degrees of freedom
AIC: -12163.35
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 1.4442 1.44423 3.7841 0.06057 .
Residuals 32 12.2131 0.38166
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
summary(both_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) +
log(Goldman_train_lemmas), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.6214 -0.9821 0.0000 0.0000 0.7569
(Dispersion Parameter for Beta regression family taken to be 0.3064)
Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.6952 on 31 degrees of freedom
AIC: -12350.04
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 3.5682 3.5682 11.6462 0.001809 **
log(Goldman_train_lemmas) 1 0.8896 0.8896 2.9036 0.098388 .
Residuals 31 9.4980 0.3064
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -12273.43
AIC(lemma_model)
[1] -12163.35
AIC(both_model)
[1] -12350.04
BIC(size_model)
[1] -12270.37
BIC(lemma_model)
[1] -12160.3
BIC(both_model)
[1] -12345.47
size_model$coefficients
(Intercept) log(Goldman_train_size)
-0.6355997 0.2861804
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
0.6611814 0.1920062
CLUZH$Goldman_scaled_acc <- (CLUZH$Goldman_acc/100 * 33 + .5) / 34
size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
summary(size_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.6764 -0.9117 0.0000 0.0000 0.7096
(Dispersion Parameter for Beta regression family taken to be 0.4547)
Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -10.9053 on 32 degrees of freedom
AIC: -10671.66
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 7.0202 7.0202 15.44 0.000427 ***
Residuals 32 14.5500 0.4547
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = betar(link = "logit"), data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.5490 -1.0794 0.0000 0.0000 0.8328
(Dispersion Parameter for Beta regression family taken to be 0.5099)
Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -12.5557 on 32 degrees of freedom
AIC: -10473.82
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 4.9185 4.9185 9.6466 0.003957 **
Residuals 32 16.3160 0.5099
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
summary(both_model)
Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) +
log(Goldman_train_lemmas), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.6752 -0.8953 0.0000 0.0000 0.7093
(Dispersion Parameter for Beta regression family taken to be 0.4671)
Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -11.0956 on 31 degrees of freedom
AIC: -10675.54
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 7.0203 7.0203 15.0303 0.0005137 ***
log(Goldman_train_lemmas) 1 0.0050 0.0050 0.0107 0.9182216
Residuals 31 14.4794 0.4671
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -10671.66
AIC(lemma_model)
[1] -10473.82
AIC(both_model)
[1] -10675.54
BIC(size_model)
[1] -10668.61
BIC(lemma_model)
[1] -10470.76
BIC(both_model)
[1] -10670.96
size_model$coefficients
(Intercept) log(Goldman_train_size)
-1.7996408 0.3963434
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
-0.5208229 0.3695156
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(Goldman_train_size, Goldman_scaled_acc)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(Goldman_train_size, size_fit)) +
geom_ribbon(aes(x = Goldman_train_size,
ymin = size_fit - 2 * size_se,
ymax = size_fit + 2 * size_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Size") +
ylab("Accuracy") +
ggtitle("Training Size vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_size_beta.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(Goldman_train_lemmas, Goldman_scaled_acc)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(Goldman_train_lemmas, lemma_fit)) +
geom_ribbon(aes(x = Goldman_train_lemmas,
ymin = lemma_fit - 2 * lemma_se,
ymax = lemma_fit + 2 * lemma_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Lemmas") +
ylab("Accuracy") +
ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
#ggsave("figures/train_lemmas_beta.pdf", device="pdf", width = 12, height=4)
all_data %>%
ggplot(aes(x = Goldman_train_size, y = acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Goldman et al. Training Size") +
ylab("Accuracy Drop") +
ggtitle("Training Size vs. Accuracy Drop")
all_data %>%
ggplot(aes(x = Goldman_train_lemmas, y = acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Goldman et al. Training Lemmas") +
ylab("Accuracy Drop") +
ggtitle("Training Lemmas vs. Accuracy Drop")
all_data %>%
ggplot(aes(x = train_lemma_diff_raw, y = acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Drop in the Number of Training Lemmas") +
ylab("Accuracy Drop") +
ggtitle("Training Lemma Drop vs. Accuracy Drop")
all_data %>%
ggplot(aes(x = log(Goldman_train_size), y = log_acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Goldman et al. Training Size, Log Scale") +
ylab("Accuracy Drop, Log Scale") +
ggtitle("Training Size vs. Accuracy Drop") +
stat_smooth(method="lm", color="grey")
all_data %>%
ggplot(aes(x = log(Goldman_train_lemmas), y = log_acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Goldman et al. Training Lemmas, Log Scale") +
ylab("Accuracy Drop, Log Scale") +
ggtitle("Training Lemmas vs. Accuracy Drop") +
stat_smooth(method="lm", color="grey")
all_data %>%
ggplot(aes(x = log(-1*train_lemma_diff_raw), y = log_acc_drop)) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_point(aes(color=Family), size=3, alpha = 0.6) +
theme_bw() +
scale_color_manual(values=c("purple","turquoise", "gold")) +
theme(plot.title = element_text(hjust=0.5, size=16)) +
xlab("Drop in Training Lemmas, Log Scale") +
ylab("Accuracy Drop, Log Scale") +
ggtitle("Lemma Drop vs. Accuracy Drop") +
stat_smooth(method="lm", color="grey")
CHRTRM$scaled_drop <- -1*(CHRTRM$Goldman_acc - CHRTRM$SIGMORPHON_acc - 2)/100
DeepSpin$scaled_drop <- -1*(DeepSpin$Goldman_acc - DeepSpin$SIGMORPHON_acc - 2)/100
CLUZH$scaled_drop <- -1*(CLUZH$Goldman_acc - CLUZH$SIGMORPHON_acc - 4)/100
correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.830738 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.798319 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.593583 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.125834 (p = 0.478255)"
[1] "Spearman's rank correlation rho: -0.032631 (p = 0.854639)"
[1] "Kendall's rank correlation tau: -0.010841 (p = 0.928903)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_size), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.233565 (p = 0.183703)"
[1] "Spearman's rank correlation rho: -0.130491 (p = 0.461985)"
[1] "Kendall's rank correlation tau: -0.067797 (p = 0.573169)"
correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.907756 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.915412 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.747098 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.095421 (p = 0.591397)"
[1] "Spearman's rank correlation rho: -0.080678 (p = 0.650137)"
[1] "Kendall's rank correlation tau: -0.052539 (p = 0.666183)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.264233 (p = 0.131023)"
[1] "Spearman's rank correlation rho: -0.196087 (p = 0.266378)"
[1] "Kendall's rank correlation tau: -0.137746 (p = 0.253461)"
correlations(log(-1*CHRTRM$train_lemma_diff_raw), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.896908 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.888906 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.726787 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(-1*DeepSpin$train_lemma_diff_raw), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.146380 (p = 0.408762)"
[1] "Spearman's rank correlation rho: -0.128783 (p = 0.467918)"
[1] "Kendall's rank correlation tau: -0.086884 (p = 0.475256)"
sprintf("")
[1] ""
correlations(log(-1*CLUZH$train_lemma_diff_raw), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.281993 (p = 0.106145)"
[1] "Spearman's rank correlation rho: -0.236036 (p = 0.178977)"
[1] "Kendall's rank correlation tau: -0.157283 (p = 0.191899)"
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(x = scaled_drop, color=Model)) +
geom_density(aes(fill= Model), alpha = 0.5) +
scale_color_manual(values=c("purple", "gold", "turquoise")) +
scale_fill_manual(values=c("purple", "gold", "turquoise")) +
theme_bw() +
ylab("Density") +
xlab("Accuracy Drop") +
ggtitle("Density of Accuracy Drop") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/drop_distribution.pdf", device="pdf", width = 8, height=5)
size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(size_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.3926 -0.3985 0.0000 0.8903 1.0690
(Dispersion Parameter for Beta regression family taken to be 0.3693)
Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: -3.3233 on 32 degrees of freedom
AIC: 1550.002
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 15.365 15.3647 41.61 2.964e-07 ***
Residuals 32 11.816 0.3693
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit
lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)
summary(lemma_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.1012 0.0000 0.6957 1.0132 1.9912
(Dispersion Parameter for Beta regression family taken to be 0.1745)
Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 16.7735 on 32 degrees of freedom
AIC: 1016.286
Number of Local Scoring Iterations: 6
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 11.8181 11.8181 67.719 2.119e-09 ***
Residuals 32 5.5846 0.1745
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CHRTRM)
summary(drop_model)
Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.5036 0.0000 0.4835 0.9504 1.5863
(Dispersion Parameter for Beta regression family taken to be 0.2471)
Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 8.804 on 32 degrees of freedom
AIC: 1193.401
Number of Local Scoring Iterations: 6
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(-1 * train_lemma_diff_raw) 1 12.6416 12.6416 51.154 4.056e-08 ***
Residuals 32 7.9081 0.2471
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$drop_fit <- res$fit
CHRTRM$drop_se <- res$se.fit
both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas) + log(-1* train_lemma_diff_raw) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(both_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 *
train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"),
data = CHRTRM)
Deviance Residuals:
Min 1Q Median 3Q Max
-1.0309 0.0000 0.6883 1.1058 2.1611
(Dispersion Parameter for Beta regression family taken to be 0.1944)
Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 20.2587 on 30 degrees of freedom
AIC: 956.8627
Number of Local Scoring Iterations: 6
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 10.9621 10.9621 56.3870 2.267e-08 ***
log(-1 * train_lemma_diff_raw) 1 0.2024 0.2024 1.0412 0.3157
log(Goldman_train_size) 1 0.1658 0.1658 0.8530 0.3631
Residuals 30 5.8323 0.1944
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
(Intercept) log(Goldman_train_size)
3.9535916 -0.5549763
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
3.9757535 -0.9047767
drop_model$coefficients
(Intercept) log(-1 * train_lemma_diff_raw)
2.8158287 -0.8161564
AIC(size_model)
[1] 1550.002
AIC(lemma_model)
[1] 1016.286
AIC(drop_model)
[1] 1193.401
BIC(size_model)
[1] 1553.055
BIC(lemma_model)
[1] 1019.338
BIC(drop_model)
[1] 1196.454
size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)
summary(size_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.7397 0.0000 0.0000 1.1084 1.4802
(Dispersion Parameter for Beta regression family taken to be 0.1433)
Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.4806 on 32 degrees of freedom
AIC: 7018.613
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 1.8557 1.85567 12.945 0.001067 **
Residuals 32 4.5871 0.14335
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit
lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
summary(lemma_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.5064 0.0000 0.0000 1.0176 1.5635
(Dispersion Parameter for Beta regression family taken to be 0.1553)
Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -14.7622 on 32 degrees of freedom
AIC: 7027.519
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 1.1735 1.17353 7.5547 0.009755 **
Residuals 32 4.9708 0.15534
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = DeepSpin)
summary(drop_model)
Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.5698 0.0000 0.0000 1.0924 1.5544
(Dispersion Parameter for Beta regression family taken to be 0.1475)
Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.6903 on 32 degrees of freedom
AIC: 6992.3
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(-1 * train_lemma_diff_raw) 1 1.5505 1.5505 10.512 0.002771 **
Residuals 32 4.7199 0.1475
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$drop_fit <- res$fit
DeepSpin$drop_se <- res$se.fit
both_model <- gam(scaled_drop ~ log(-1*train_lemma_diff_raw) + log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
summary(both_model)
Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw) +
log(Goldman_train_size) + log(Goldman_train_lemmas), family = betar(link = "logit"),
data = DeepSpin)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.8752 0.0000 0.0000 0.9393 1.5220
(Dispersion Parameter for Beta regression family taken to be 0.1435)
Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.7918 on 30 degrees of freedom
AIC: 6982.896
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(-1 * train_lemma_diff_raw) 1 1.6345 1.63447 11.3933 0.002052 **
log(Goldman_train_size) 1 0.1767 0.17666 1.2314 0.275945
log(Goldman_train_lemmas) 1 0.3227 0.32272 2.2495 0.144106
Residuals 30 4.3038 0.14346
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
(Intercept) log(Goldman_train_size)
-0.2192908 -0.2378525
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
-1.0371551 -0.2098211
drop_model$coefficients
(Intercept) log(-1 * train_lemma_diff_raw)
-1.0666528 -0.2533969
AIC(size_model)
[1] 7018.613
AIC(lemma_model)
[1] 7027.519
AIC(drop_model)
[1] 6992.3
BIC(size_model)
[1] 7021.666
BIC(lemma_model)
[1] 7030.572
BIC(drop_model)
[1] 6995.352
size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)
summary(size_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.838 0.000 0.000 1.005 1.417
(Dispersion Parameter for Beta regression family taken to be 0.1677)
Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 1.805 on 32 degrees of freedom
AIC: 5855.958
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_size) 1 2.2528 2.25278 13.431 0.000889 ***
Residuals 32 5.3675 0.16774
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
summary(lemma_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.6502 0.0000 0.3396 1.0907 1.4213
(Dispersion Parameter for Beta regression family taken to be 0.1501)
Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.7868 on 32 degrees of freedom
AIC: 5823.695
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 2.5131 2.51310 16.738 0.0002709 ***
Residuals 32 4.8047 0.15015
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CLUZH)
summary(drop_model)
Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.841 0.000 0.000 1.094 1.376
(Dispersion Parameter for Beta regression family taken to be 0.1516)
Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.8613 on 32 degrees of freedom
AIC: 5831.558
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(-1 * train_lemma_diff_raw) 1 2.5820 2.58196 17.027 0.0002453 ***
Residuals 32 4.8526 0.15164
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$drop_fit <- res$fit
CLUZH$drop_se <- res$se.fit
both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas) + log(-1*train_lemma_diff_raw) + log(Goldman_train_size) , family=betar(link="logit"), data = CLUZH)
summary(both_model)
Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 *
train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"),
data = CLUZH)
Deviance Residuals:
Min 1Q Median 3Q Max
-0.7097 0.0000 0.3350 1.0758 1.4194
(Dispersion Parameter for Beta regression family taken to be 0.1584)
Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 4.0005 on 30 degrees of freedom
AIC: 5837.224
Number of Local Scoring Iterations: 2
Anova for Parametric Effects
Df Sum Sq Mean Sq F value Pr(>F)
log(Goldman_train_lemmas) 1 2.5226 2.52261 15.9285 0.0003913 ***
log(-1 * train_lemma_diff_raw) 1 0.0524 0.05242 0.3310 0.5693799
log(Goldman_train_size) 1 0.0128 0.01279 0.0808 0.7782086
Residuals 30 4.7511 0.15837
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
(Intercept) log(Goldman_train_size)
0.1286658 -0.2387694
lemma_model$coefficients
(Intercept) log(Goldman_train_lemmas)
-0.328039 -0.287939
drop_model$coefficients
(Intercept) log(-1 * train_lemma_diff_raw)
-0.5445538 -0.3002098
AIC(size_model)
[1] 5855.958
AIC(lemma_model)
[1] 5823.695
AIC(drop_model)
[1] 5831.558
BIC(size_model)
[1] 5859.011
BIC(lemma_model)
[1] 5826.747
BIC(drop_model)
[1] 5834.611
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(Goldman_train_size, -1*scaled_drop)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(Goldman_train_size, -1*size_fit)) +
geom_ribbon(aes(x = Goldman_train_size,
ymin = -1*(size_fit - 2 * size_se),
ymax = -1*(size_fit + 2 * size_se),
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Size") +
ylab("Scaled Accuracy Drop") +
ggtitle("Training Size vs. Accuracy Drop") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_size_drop.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(Goldman_train_lemmas, -1*scaled_drop)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(Goldman_train_lemmas, -1*lemma_fit)) +
geom_ribbon(aes(x = Goldman_train_lemmas,
ymin = -1*(lemma_fit - 2 * lemma_se),
ymax = -1*(lemma_fit + 2 * lemma_se),
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Lemmas") +
ylab("Scaled Accuracy Drop") +
ggtitle("Training Lemmas vs. Accuracy Drop") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_lemmas_drop.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(-1*train_lemma_diff_raw, -1*scaled_drop)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(-1*train_lemma_diff_raw, -1*drop_fit)) +
geom_ribbon(aes(x = -1*train_lemma_diff_raw,
ymin = -1*(drop_fit - 2 * drop_se),
ymax = -1*(drop_fit + 2 * drop_se),
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Drop in Lemmas from SIGMORPHON to Goldman et al.") +
ylab("Scaled Accuracy Drop") +
ggtitle("Training Lemma Drop vs. Accuracy Drop") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/lemma_drop_drop.pdf", device="pdf", width = 12, height=4)
size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size), data = CHRTRM)
summary(size_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size),
data = CHRTRM)
Residuals:
Min 1Q Median 3Q Max
-2.12468 -0.31146 0.01354 0.48987 1.86432
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -4.58098 0.65386 -7.006 6.12e-08 ***
log(Goldman_train_size) 0.42338 0.07346 5.763 2.16e-06 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.9212 on 32 degrees of freedom
Multiple R-squared: 0.5093, Adjusted R-squared: 0.494
F-statistic: 33.22 on 1 and 32 DF, p-value: 2.156e-06
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit
lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas),
data = CHRTRM)
Residuals:
Min 1Q Median 3Q Max
-2.5247 -0.5142 0.3007 0.7480 1.1648
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -3.21974 0.47613 -6.762 1.22e-07 ***
log(Goldman_train_lemmas) 0.40297 0.07828 5.148 1.29e-05 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.9727 on 32 degrees of freedom
Multiple R-squared: 0.453, Adjusted R-squared: 0.4359
F-statistic: 26.5 on 1 and 32 DF, p-value: 1.293e-05
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM)
summary(both_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), data = CHRTRM)
Residuals:
Min 1Q Median 3Q Max
-2.18137 -0.36728 0.06201 0.54080 1.74301
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -4.3744 0.7339 -5.960 1.37e-06 ***
log(Goldman_train_lemmas) 0.1066 0.1656 0.644 0.5246
log(Goldman_train_size) 0.3292 0.1641 2.006 0.0537 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.9298 on 31 degrees of freedom
Multiple R-squared: 0.5158, Adjusted R-squared: 0.4846
F-statistic: 16.51 on 2 and 31 DF, p-value: 1.312e-05
AIC(size_model)
[1] 94.84671
AIC(lemma_model)
[1] 98.5432
AIC(both_model)
[1] 96.39553
anova(both_model, lemma_model)
Analysis of Variance Table
Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
Res.Df RSS Df Sum of Sq F Pr(>F)
1 31 26.799
2 32 30.276 -1 -3.4769 4.022 0.05371 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size), data = DeepSpin)
summary(size_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size),
data = DeepSpin)
Residuals:
Min 1Q Median 3Q Max
-1.09702 -0.07001 0.01215 0.18962 0.42460
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.77218 0.22958 -3.363 0.00201 **
log(Goldman_train_size) 0.06590 0.02579 2.555 0.01558 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.3234 on 32 degrees of freedom
Multiple R-squared: 0.1694, Adjusted R-squared: 0.1435
F-statistic: 6.527 on 1 and 32 DF, p-value: 0.01558
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit
lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas),
data = DeepSpin)
Residuals:
Min 1Q Median 3Q Max
-1.28791 -0.03124 0.06302 0.20218 0.30658
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.42363 0.16866 -2.512 0.0173 *
log(Goldman_train_lemmas) 0.03873 0.02773 1.396 0.1722
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.3446 on 32 degrees of freedom
Multiple R-squared: 0.05744, Adjusted R-squared: 0.02799
F-statistic: 1.95 on 1 and 32 DF, p-value: 0.1722
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin)
summary(both_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), data = DeepSpin)
Residuals:
Min 1Q Median 3Q Max
-0.89596 -0.05727 0.06613 0.13517 0.53952
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.96792 0.24663 -3.925 0.00045 ***
log(Goldman_train_lemmas) -0.10099 0.05567 -1.814 0.07936 .
log(Goldman_train_size) 0.15516 0.05516 2.813 0.00844 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.3125 on 31 degrees of freedom
Multiple R-squared: 0.2491, Adjusted R-squared: 0.2007
F-statistic: 5.143 on 2 and 31 DF, p-value: 0.01178
AIC(size_model)
[1] 23.67421
AIC(lemma_model)
[1] 27.97414
AIC(both_model)
[1] 22.24389
anova(both_model, lemma_model)
Analysis of Variance Table
Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
Res.Df RSS Df Sum of Sq F Pr(>F)
1 31 3.0265
2 32 3.7992 -1 -0.77262 7.9137 0.00844 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size), data = CLUZH)
summary(size_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size),
data = CLUZH)
Residuals:
Min 1Q Median 3Q Max
-2.01946 -0.16571 -0.01182 0.38359 0.86925
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -1.65335 0.40383 -4.094 0.000269 ***
log(Goldman_train_size) 0.15234 0.04537 3.358 0.002041 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.5689 on 32 degrees of freedom
Multiple R-squared: 0.2605, Adjusted R-squared: 0.2374
F-statistic: 11.27 on 1 and 32 DF, p-value: 0.002041
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas),
data = CLUZH)
Residuals:
Min 1Q Median 3Q Max
-2.4300 -0.1431 0.1326 0.3918 0.6468
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.99545 0.29911 -3.328 0.00221 **
log(Goldman_train_lemmas) 0.11548 0.04918 2.348 0.02521 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.6111 on 32 degrees of freedom
Multiple R-squared: 0.147, Adjusted R-squared: 0.1203
F-statistic: 5.514 on 1 and 32 DF, p-value: 0.02521
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH)
summary(both_model)
Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), data = CLUZH)
Residuals:
Min 1Q Median 3Q Max
-1.80791 -0.16141 0.01489 0.28101 0.99017
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -1.8593 0.4484 -4.147 0.000242 ***
log(Goldman_train_lemmas) -0.1063 0.1012 -1.050 0.301858
log(Goldman_train_size) 0.2463 0.1003 2.456 0.019855 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.568 on 31 degrees of freedom
Multiple R-squared: 0.2859, Adjusted R-squared: 0.2399
F-statistic: 6.206 on 2 and 31 DF, p-value: 0.005408
AIC(size_model)
[1] 62.07644
AIC(lemma_model)
[1] 66.93316
AIC(both_model)
[1] 62.88837
anova(both_model, lemma_model)
Analysis of Variance Table
Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
Res.Df RSS Df Sum of Sq F Pr(>F)
1 31 10.003
2 32 11.949 -1 -1.9462 6.0317 0.01986 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(log(Goldman_train_size), log(Goldman_scaled_acc))) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(log(Goldman_train_size), size_fit)) +
geom_ribbon(aes(x = log(Goldman_train_size),
ymin = size_fit - 2 * size_se,
ymax = size_fit + 2 * size_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Size, Log Scale") +
ylab("Accuracy") +
ggtitle("Training Size vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_size_lm.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(log(Goldman_train_lemmas), log(Goldman_scaled_acc))) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) +
geom_ribbon(aes(x = log(Goldman_train_lemmas),
ymin = lemma_fit - 2 * lemma_se,
ymax = lemma_fit + 2 * lemma_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Lemmas, Log Scale") +
ylab("Accuracy") +
ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_lemmas_lm.pdf", device="pdf", width = 12, height=4)
size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size), data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial,
data = CHRTRM)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -5.0645 1.9139 -2.646 0.00814 **
log(Goldman_train_size) 0.6567 0.2284 2.875 0.00404 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 18.0726 on 33 degrees of freedom
Residual deviance: 6.3867 on 32 degrees of freedom
AIC: 31.886
Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit
lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = binomial, data = CHRTRM)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -3.8352 1.6005 -2.396 0.0166 *
log(Goldman_train_lemmas) 0.8112 0.3161 2.566 0.0103 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 18.0726 on 33 degrees of freedom
Residual deviance: 5.6775 on 32 degrees of freedom
AIC: 28.439
Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), family = binomial, data = CHRTRM)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -4.5197 1.9676 -2.297 0.0216 *
log(Goldman_train_lemmas) 0.5324 0.5415 0.983 0.3255
log(Goldman_train_size) 0.2584 0.4329 0.597 0.5507
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 18.0726 on 33 degrees of freedom
Residual deviance: 5.3129 on 31 degrees of freedom
AIC: 30.767
Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 31.88575
AIC(lemma_model)
[1] 28.4393
AIC(both_model)
[1] 30.76683
anova(both_model, lemma_model)
Analysis of Deviance Table
Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1 31 5.3129
2 32 5.6775 -1 -0.36453 0.546
size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size), data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial,
data = DeepSpin)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -0.5417 1.9087 -0.284 0.777
log(Goldman_train_size) 0.2811 0.2380 1.181 0.238
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 8.0760 on 33 degrees of freedom
Residual deviance: 6.5801 on 32 degrees of freedom
AIC: 19.053
Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit
lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = binomial, data = DeepSpin)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 0.7911 1.3545 0.584 0.559
log(Goldman_train_lemmas) 0.1786 0.2453 0.728 0.466
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 8.0760 on 33 degrees of freedom
Residual deviance: 7.5112 on 32 degrees of freedom
AIC: 21.031
Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), family = binomial, data = DeepSpin)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -0.9913 1.9715 -0.503 0.615
log(Goldman_train_lemmas) -0.3398 0.5003 -0.679 0.497
log(Goldman_train_size) 0.5596 0.4763 1.175 0.240
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 8.0760 on 33 degrees of freedom
Residual deviance: 6.1213 on 31 degrees of freedom
AIC: 19.851
Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 19.05313
AIC(lemma_model)
[1] 21.03097
AIC(both_model)
[1] 19.85117
anova(both_model, lemma_model)
Analysis of Deviance Table
Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1 31 6.1213
2 32 7.5112 -1 -1.39 0.2384
size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size), data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial,
data = CLUZH)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -1.8475 1.8112 -1.020 0.3077
log(Goldman_train_size) 0.4071 0.2317 1.757 0.0789 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 12.9162 on 33 degrees of freedom
Residual deviance: 9.3231 on 32 degrees of freedom
AIC: 24.943
Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas),
family = binomial, data = CLUZH)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -0.5165 1.3129 -0.393 0.694
log(Goldman_train_lemmas) 0.3758 0.2575 1.460 0.144
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 12.916 on 33 degrees of freedom
Residual deviance: 10.291 on 32 degrees of freedom
AIC: 26.417
Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)
Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) +
log(Goldman_train_size), family = binomial, data = CLUZH)
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -1.89248 1.89135 -1.001 0.317
log(Goldman_train_lemmas) -0.03862 0.48110 -0.080 0.936
log(Goldman_train_size) 0.43737 0.44328 0.987 0.324
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 12.9162 on 33 degrees of freedom
Residual deviance: 9.3167 on 31 degrees of freedom
AIC: 27.012
Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 24.94273
AIC(lemma_model)
[1] 26.41689
AIC(both_model)
[1] 27.01204
anova(both_model, lemma_model)
Analysis of Deviance Table
Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1 31 9.3167
2 32 10.2914 -1 -0.97468 0.3235
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(log(Goldman_train_size), Goldman_scaled_acc)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(log(Goldman_train_size), size_fit)) +
geom_ribbon(aes(x = log(Goldman_train_size),
ymin = size_fit - 2 * size_se,
ymax = size_fit + 2 * size_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Size, Log Scale") +
ylab("Accuracy") +
ggtitle("Training Size vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_size_logistic.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>%
ggplot(aes(log(Goldman_train_lemmas), Goldman_scaled_acc)) +
geom_point(aes(color=Family), size = 3, alpha = 0.6) +
scale_color_manual(values=c("turquoise", "purple", "gold")) +
facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) +
geom_ribbon(aes(x = log(Goldman_train_lemmas),
ymin = lemma_fit - 2 * lemma_se,
ymax = lemma_fit + 2 * lemma_se,
), fill ="grey",
alpha = 0.4) +
theme_bw() +
xlab("Goldman Training Lemmas, Log Scale") +
ylab("Accuracy") +
ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") +
theme(plot.title = element_text(hjust=0.5, size=16))
ggsave("figures/train_lemmas_logistic.pdf", device="pdf", width = 12, height=4)